# -*- coding: utf-8 -*-
import re
import sys
reload(sys)
sys.setdefaultencoding('utf-8')
from scrapy.http import Request as sreq
from scrapy.http import FormRequest as f
from girl.items.xvideo import *
from scrapy_redis.spiders import RedisCrawlSpider
from datetime import *
import redis
from scrapy.spiders import *
from scrapy.linkextractors import LinkExtractor
import platform

class XvideoSpider(RedisCrawlSpider):
    name = "xvideo"
    start_urls = [
       "https://www.xvideos.com/tags"
    ]

    pool = redis.ConnectionPool(host='localhost', port=6379)
    conn = redis.Redis(connection_pool=pool)
    conn.sadd("xvideo:start_urls", *start_urls)
    custom_settings = {
        # 'RETRY_ENABLED' : False,
        # "DOWNLOAD_TIMEOUT": 380,
        # "DOWNLOAD_DELAY ": 0.25,
        # "CONCURRENT_REQUESTS": 1,
        # "SCHEDULER_PERSIST" : False,
        # "DUPEFILTER_CLASS" : ""
    }
    if platform.system() != 'Darwin':
        time = datetime.now().strftime('%Y-%m-%d-%H-%M')
        custom_settings["LOG_FILE"] = "/mnt/scrapy/crawler/%s-%s.log" % (name,time)
    rules = (
        Rule(LinkExtractor(
        allow=('tags'),
        restrict_xpaths="//ul[@id='tags']//a"),
        follow=True,
        callback='parse1'
        ),
        Rule(LinkExtractor(
        # allow=('new'),
        restrict_xpaths="//div[@class='pagination ']//a"),
        follow=True,
        callback='parse1'
        )
        # Rule(LinkExtractor(
        # allow=('video'),
        # restrict_css=".mozaique .thumb a"),
        # ),
        # callback='video')
    )
    # def start_requests(self):
    #     for url in self.start_urls:
    #         yield Request(url, dont_filter=True)

    def parse1(self, response):
        for x in response.css(".mozaique .thumb a"):
            url = x.css("::attr('href')").extract_first()
            thumb = x.css("img::attr(src)").extract_first()
            yield sreq(response.urljoin(url),callback=self.video,meta={"thumb":thumb})

    def video(self, response):
        x = response.css
        item = videoItem()
        item["title"] = x(".page-title ::text").extract_first()
        item['tag'] = x('.video-metadata .btn.btn-default::text').extract()
        item["duration"] = x('.duration ::text').extract_first()
        item["date"] = str(date.today())
        item["url"] = response.url
        item["thumb"] = re.findall(r"setThumbUrl.*\('(.*)'\)",response.body)
        yield item


